package 大数据比赛样题_2020

import org.apache.spark.sql
import org.apache.spark.sql.SparkSession

object t01 {

  def main(args: Array[String]): Unit = {

    // 配置环境
    val spark = new sql.SparkSession.Builder()
      .appName("t01")
      .master("local[6]")
      .getOrCreate()
    // 导入隐式转换、函数
    import spark.implicits._
    import org.apache.spark.sql.functions._

    // 读取数据集
    val students = spark.sparkContext.textFile("hdfs://192.168.64.129:9000/user/root/testdata/students.txt")
    students.collect().foreach(println(_))
    // 提取信息聚合
    val class_students = students.map(item=>(item.split(",")(4),1))
      .reduceByKey(_+_)
      .sortByKey()
      class_students.collect().foreach(println(_))
    // 转换形态
    val result = class_students.map(item=>Array(item._1,item._2).mkString(","))
    result.collect().foreach(println(_))

  }
}
